#include <stdio.h>
#include <string.h>


#define MAX_LINE_LENGTH 1023
#define HASH_PRIME 366239

#define MIN_OCCURENCE 2000
#define MIN_YEAR 1970

inline int good_word(char* s) {
    int begin, end, i, apos, minus;
    for (begin = 0; s[begin];) {
        while (s[begin] == ' ') begin++;
        end = begin;
        while (s[end] && s[end] != ' ') end++;
        apos = 0;
        minus = 0;
        for (i = begin; i < end; i++) {
            if ('a' <= s[i] && s[i] <= 'z') continue;
            if (s[i] == ' ') continue;
            if (s[i] == '\'' && (!s[i + 1] || s[i + 1] == ' ')) return 0;
            if (s[i] == '\'') { apos++; continue; }
            if (s[i] == '-' && (!s[i + 1] || s[i + 1] == ' ')) return 0;
            if (s[i] == '-') { minus++; continue; }
            return 0;
        }
        if (apos > 1 || minus > 1 || apos + minus == end - begin) return 0;
        begin = end;
    }
    return 1;
}

int main(int argc, char* argv[]) {
    if (argc < 2) return 1 + 0 * printf("pystsch: Completion Project corpus handler\n\
Walks through a Google corpus file, filtering n-grams which did appear frequently in recent years\n\
---\n\
Usage: %s <filename>\n\
filename -- string, name of Google corpus file to parse\n\
---\n\
Results will appear in \"out\" file\n", argv[0]);
    char temp[MAX_LINE_LENGTH + 1];
    char word[MAX_LINE_LENGTH + 1];
    int last_hash, cur_hash, i, y;
    long long occ, k;

    freopen(argv[1], "r", stdin);
    freopen("out", "w", stdout);
    last_hash = 0;
    while (gets(temp)) {
        for (i = 0, cur_hash = 0; temp[i] != '\t'; i++) {
            cur_hash = cur_hash * HASH_PRIME + temp[i];
        }
        if (cur_hash != last_hash) {
            if (occ > MIN_OCCURENCE) printf("%s\t%lld\n", word, occ);
            occ = 0;
            last_hash = cur_hash;
        }
        memcpy(word, temp, i * sizeof(word[0]));
        word[i] = 0;
        if (!good_word(word)) continue;
        i++;
        for (y = 0; temp[i] != '\t'; i++) {
            y = y * 10 + (temp[i] - '0');
        }
        if (y >= MIN_YEAR) {
            i++;
            for (k = 0; temp[i] != '\t'; i++) {
                k = k * 10 + (temp[i] - '0');
            }
            occ += k;
        }
    }
    return 0;
}

